#coding=utf-8
from convert_util import *

from task201904util import  *

input = r'D:\sfxy191\train_data'
out1 = r'D:\sfxy191\train_per1'
out2 = r'D:\sfxy191\train_per2'

SENTENCES = 10200

def raw_to_html():
    i = 0;
    fw = open(r'{}{}.html'.format(input, i), encoding='utf-8', mode="w")
    with open(input, encoding='utf-8') as f:
        sentences = f.read().split("\n\n")
        person_sorter = PersonSorter(out1, out2)
        for s in sentences :
            html_content = parse_raw_out(s)
            fw.write("{}&nbsp;&nbsp;&nbsp;&nbsp;{}<br></br>\n".format(i, html_content))
            i += 1
            person_sorter.process_one(html_content, i)
            if i % SENTENCES == 0:
                print(i)
            #     break
                fw.close()
                fw = open(r'{}{}.html'.format(input, i), encoding='utf-8', mode="w")
        print(i)
        person_sorter.close()
        person_sorter.person_reset_id()

    fw.close()

def add_jobs(file_in, file_out, jobs):
    fw = open(file_out, encoding= "utf-8", mode="w")
    with open(file_in, encoding='utf-8') as f:
        for line in f:
            if line[:3] == '454':
                print(line)
            for job in jobs:
                # if job == '副教授':
                #     print(job)
                # if job == "教授":
                #     print(job)
                line = color_new_term(line, job, "JOB")
            fw.write(line)
    fw.close()

def add_job2_files(input, *term_files):
    blocks = list(range(0, 5))
    term_set = set()
    for tf in term_files:
        with open(tf, encoding="utf-8") as f:
            term_set.update([x.strip() for x in f.read().split() if len(x.strip()) > 1])
    terms = sorted(term_set, key= lambda x:-len(x))
    for b in blocks:
        print("processing block:{}".format(b))
        add_jobs((input +"{}.html").format(b * SENTENCES), (input +"{}-job.html").format(b), terms)



if __name__ == "__main__":
    # add_jobs(input + "0.html", input + "0-job.html", ["藏书家","作家","教师"])
    #raw_to_html()
    # add_job2_files(input, "职业.txt", "职务.txt")
    extract_persons_id(input, input + "_persons")
    extract_persons2(input,input + "_persons")